package SparkSQL

import org.apache.spark.rdd.RDD
import org.apache.spark.sql.{DataFrame, Dataset, Row, SparkSession}

object transformation {
  def main(args: Array[String]): Unit = {
    //TODO 创建运行环境
    val spark = SparkSession.builder()
      .master("local[*]")
      .appName("transformation")
      .getOrCreate()
    //隐式转换
    import spark.implicits._
    //TODO 逻辑执行

    //DataFrame
    val df = spark.read.json("data/user.json")
    //df.show()
    //DataFrame  =>  SQL
    df.createOrReplaceTempView("user")
    spark.sql("select * from user").show()
    //DataFrame  =>  DSL
    df.select('age, 'username).show()
    df.select($"age" + 1).show()
    df.select('age + 1).show()
    //DatSet
    //DataFrame可以使用的方法DataSet都是可以使用的
    val ds: Dataset[Int] = Seq(1, 2, 3).toDS()
    ds.show()
    //RDD <=> DataFrame
    val rdd: RDD[(Int, String, Int)] = spark.sparkContext.makeRDD(List((1, "zhangsan", 20), (2, "lisi", 21), (3, "ww", 20)))
    val dataFrame: DataFrame = rdd.toDF("id", "name", "age")
    val rowRDD: RDD[Row] = dataFrame.rdd
    //DataFrame <=> DataSet
    val dataset = df.as[user]
    val dataFrame1 = ds.toDF()
    //RDD <=> DataSet
    val dataset2 = rdd.map {
      case (id, name, age) => {
        user(id, name, age)
      }
    }.toDS()
    val userRDD = dataset2.rdd
    //TODO 关闭环境
    spark.close()
  }

  case class user(id: Int, name: String, age: Int)
}
